import pymysql as MySQLdb
import json
from nltk.stem.lancaster import LancasterStemmer
import re

st = LancasterStemmer()
pattern = r'[^(\x20-\x7F)]'
conn = MySQLdb.connect(host='192.168.1.40',user='root',passwd='password',db='citeseerx',port=3306)
cur = conn.cursor()
cur.execute("select count(*) from cluster")
count = cur.fetchone()[0]
print("Count: " + str(count))
limit_num = 0
while limit_num < count:
    cur.execute("select id, ctitle from cluster limit " + str(limit_num) + ", 50")
    arow = cur.fetchone()
    while type(arow) != type(None):
        try:
            arow = (str(arow[0]), re.sub(pattern, '', arow[1]).lower().split())
        except:
            arow = (str(arow[0]), [])
        for word in set(arow[1]):
            stemmed = st.stem(word)
            stemmed = stemmed.replace(',', '').replace('.', '').replace('(', '')
            stemmed = stemmed.replace(')', '').replace('[', '').replace(']', '')
            stemmed = stemmed.replace('?', '').replace('/', '').replace('\\','')
            stemmed = stemmed.replace('\'','').replace('\"','')
            try:
                f = open("./wordsTable/%s.json" % stemmed, 'r')
                tmpjson = json.loads(f.read())
                f.close()
                #print("appending to current json")
                tmpjson.append(arow[0])
                f = open("./wordsTable/%s.json" % stemmed, 'w')
                f.write(json.dumps(tmpjson))
                f.close()
            except:
                #print("creating new json ...")
                f = open("./wordsTable/%s.json" % stemmed, 'w')
                f.write("[ \"%s\" ]" % arow[0])
                f.close()
        arow = cur.fetchone()
    limit_num += 50
    print(limit_num)
print("Done")